{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow 1.0 API使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.1\n",
      "sys.version_info(major=3, minor=6, micro=8, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.3\n",
      "numpy 1.16.4\n",
      "pandas 0.24.2\n",
      "sklearn 0.22.1\n",
      "tensorflow 1.13.1\n",
      "tensorflow._api.v1.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "# 导入\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据读取--fashion_mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 28, 28) (5000,)\n",
      "(55000, 28, 28) (55000,)\n",
      "(10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "# 导入数据集fashion mnist\n",
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(x_train_all,y_train_all),(x_test,y_test) = fashion_mnist.load_data()\n",
    "x_valid,x_train = x_train_all[:5000],x_train_all[5000:]\n",
    "y_valid,y_train = y_train_all[:5000],y_train_all[5000:]\n",
    "\n",
    "print(x_valid.shape,y_valid.shape)\n",
    "print(x_train.shape,y_train.shape)\n",
    "print(x_test.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据归一化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "x_train_scaled = scaler.fit_transform(\n",
    "    x_train.astype(np.float32).reshape(-1,1)).reshape(-1, 28 * 28)\n",
    "x_valid_scaled = scaler.transform(\n",
    "    x_valid.astype(np.float32).reshape(-1,1)).reshape(-1, 28 * 28)\n",
    "x_test_scaled = scaler.transform(\n",
    "    x_test.astype(np.float32).reshape(-1,1)).reshape(-1, 28 * 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0231433 -0.8105136\n"
     ]
    }
   ],
   "source": [
    "print(np.max(x_train_scaled),np.min(x_train_scaled))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算图构建--实现全连接网络\n",
    "使用基础的语法，实现图像分类的例子\n",
    "\n",
    "先构建图，再传入数据，要用占位符来顶替数据的位置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-8-d810a880ea1b>:13: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From f:\\condaenv\\tfgpu\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From f:\\condaenv\\tfgpu\\lib\\site-packages\\tensorflow\\python\\ops\\losses\\losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = [100, 100]\n",
    "class_num = 10\n",
    "\n",
    "# 占位符，顶替数据位置\n",
    "x = tf.placeholder(tf.float32, [None, 28 * 28])\n",
    "y = tf.placeholder(tf.int64, [None])\n",
    "\n",
    "# 定义输入层\n",
    "input_for_next_layer = x\n",
    "# 定义隐藏层\n",
    "for hidden_unit in hidden_units:\n",
    "    input_for_next_layer = tf.layers.dense(\n",
    "        input_for_next_layer, hidden_unit, activation=tf.nn.relu)\n",
    "    \n",
    "# 输出层\n",
    "# last_hidden_output * W (logits) -> softmax -> prob概率\n",
    "# 1. logit -> softmax -> prob\n",
    "# 2. labels -> one_hot\n",
    "# 3. calculate cross entropy\n",
    "logits = tf.layers.dense(input_for_next_layer, class_num)\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits)\n",
    "\n",
    "# 得到预测值\n",
    "prediction = tf.argmax(logits, 1)\n",
    "correct_prediction = tf.equal(prediction, y)\n",
    "# get accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))\n",
    "\n",
    "# 运行一次train_op，网络就训练一次\n",
    "train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Placeholder:0\", shape=(?, 784), dtype=float32)\n",
      "Tensor(\"dense_2/BiasAdd:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(x)\n",
    "print(logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练\n",
    "session 是客户端（python）和C++运行时的连接，打开会话，就可以运行图了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] epoch:0, step:2749, loss:0.38786, accuracy:0.80\t[Valid] acc:0.86\n",
      "[Train] epoch:1, step:2749, loss:0.23071, accuracy:0.85\t[Valid] acc:0.87\n",
      "[Train] epoch:2, step:2749, loss:0.19158, accuracy:0.95\t[Valid] acc:0.88\n",
      "[Train] epoch:3, step:2749, loss:0.15562, accuracy:0.90\t[Valid] acc:0.88\n",
      "[Train] epoch:4, step:2749, loss:0.20032, accuracy:0.90\t[Valid] acc:0.88\n",
      "[Train] epoch:5, step:2749, loss:0.12497, accuracy:0.90\t[Valid] acc:0.88\n",
      "[Train] epoch:6, step:2749, loss:0.10159, accuracy:0.95\t[Valid] acc:0.88\n",
      "[Train] epoch:7, step:2749, loss:0.07915, accuracy:0.95\t[Valid] acc:0.88\n",
      "[Train] epoch:8, step:2749, loss:0.04988, accuracy:1.00\t[Valid] acc:0.88\n",
      "[Train] epoch:9, step:2749, loss:0.10262, accuracy:0.90\t[Valid] acc:0.88\n"
     ]
    }
   ],
   "source": [
    "# 初始化\n",
    "init = tf.global_variables_initializer()\n",
    "batch_size = 20\n",
    "epochs = 10\n",
    "train_steps_per_epoch = x_train_scaled.shape[0] // batch_size\n",
    "valid_steps = x_valid_scaled.shape[0] // batch_size\n",
    "\n",
    "# 验证\n",
    "def eval_with_sess(sess, x, y, accuracy, images, labels, batch_size):\n",
    "    eval_steps = images.shape[0] // batch_size\n",
    "    eval_accuracies = []\n",
    "    for step in range(eval_steps):\n",
    "        batch_data = images[step * batch_size : (step + 1) * batch_size]\n",
    "        batch_label = labels[step * batch_size : (step + 1) * batch_size]\n",
    "        accuracy_val = sess.run(accuracy, feed_dict = {\n",
    "            x: batch_data,\n",
    "            y: batch_label\n",
    "        })\n",
    "        eval_accuracies.append(accuracy_val)\n",
    "    return np.mean(eval_accuracies)\n",
    "\n",
    "# 创建会话\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(epochs):\n",
    "        for step in range(train_steps_per_epoch):\n",
    "            # 取数据\n",
    "            batch_data = x_train_scaled[step * batch_size : (step + 1) * batch_size]\n",
    "            batch_label = y_train[step * batch_size : (step + 1) * batch_size]\n",
    "            # 训练\n",
    "            loss_val, accuracy_val, _ = sess.run(\n",
    "                [loss, accuracy, train_op],\n",
    "                feed_dict = {\n",
    "                    x: batch_data,\n",
    "                    y: batch_label\n",
    "                })\n",
    "            print('\\r[Train] epoch:%d, step:%d, loss:%3.5f, accuracy:%2.2f' % (\n",
    "                epoch, step, loss_val, accuracy_val), end='')\n",
    "        # 验证\n",
    "        valid_accuracy = eval_with_sess(sess, x, y, accuracy,\n",
    "                                        x_valid_scaled, y_valid, batch_size)\n",
    "        print('\\t[Valid] acc:%2.2f' % (valid_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset构建，再建模训练\n",
    "\n",
    "### Dataset构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.asarray(y_train, dtype=np.int64)\n",
    "y_valid = np.asarray(y_valid, dtype=np.int64)\n",
    "y_test = np.asarray(y_test, dtype=np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dataset(images, labels, epochs, batch_size, shuffle = True):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((images, labels))\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(10000)\n",
    "    dataset = dataset.repeat(epochs).batch(batch_size)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test tf1.0 中 dataset是不可迭代的，会报错\n",
    "batch_size = 20\n",
    "epochs = 10\n",
    "dataset = make_dataset(x_train_scaled, y_train, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "# 报错，只能在eager mode下使用\n",
    "# for data, label in dataset.take(1):\n",
    "#     print(data)\n",
    "#     print(label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dataset.make_one_shot_iterator 的特点：\n",
    "1. auto initialization\n",
    "2. can't be re-initialized.\n",
    "   make_initializable_iterator can be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 784)\n",
      "(20,)\n"
     ]
    }
   ],
   "source": [
    "# tf1.0 中正确使用\n",
    "batch_size = 20\n",
    "epochs = 10\n",
    "dataset = make_dataset(x_train_scaled, y_train, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "dataset_iter = dataset.make_one_shot_iterator()\n",
    "x, y = dataset_iter.get_next()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    x_val, y_val = sess.run([x, y])\n",
    "    print(x_val.shape)\n",
    "    print(y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建模训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_units = [100, 100]\n",
    "class_num = 10\n",
    "\n",
    "# 定义输入层\n",
    "input_for_next_layer = x\n",
    "# 定义隐藏层\n",
    "for hidden_unit in hidden_units:\n",
    "    input_for_next_layer = tf.layers.dense(\n",
    "        input_for_next_layer, hidden_unit, activation=tf.nn.relu)\n",
    "    \n",
    "# 输出层\n",
    "logits = tf.layers.dense(input_for_next_layer, class_num)\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits)\n",
    "\n",
    "# 得到预测值\n",
    "prediction = tf.argmax(logits, 1)\n",
    "correct_prediction = tf.equal(prediction, y)\n",
    "# get accuracy\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))\n",
    "\n",
    "# 运行一次train_op，网络就训练一次\n",
    "train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"IteratorGetNext_1:0\", shape=(?, 784), dtype=float32)\n",
      "Tensor(\"dense_8/BiasAdd:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(x)\n",
    "print(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] epoch:9, step:2749, loss:0.33168, accuracy:0.90"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "train_steps_per_epoch = x_train_scaled.shape[0] // batch_size\n",
    "\n",
    "# 创建会话\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(epochs):\n",
    "        for step in range(train_steps_per_epoch):\n",
    "            # 训练\n",
    "            loss_val, accuracy_val, _ = sess.run(\n",
    "                [loss, accuracy, train_op])\n",
    "            print('\\r[Train] epoch:%d, step:%d, loss:%3.5f, accuracy:%2.2f' % (\n",
    "                epoch, step, loss_val, accuracy_val), end='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 另一种Dataset构建方式（推荐）\n",
    "### Dataset构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 784)\n",
      "(20,)\n",
      "(20, 784)\n",
      "(20,)\n"
     ]
    }
   ],
   "source": [
    "batch_size = 20\n",
    "epochs = 10\n",
    "\n",
    "# 构建dataset\n",
    "images_placeholder = tf.placeholder(tf.float32, [None, 28 * 28])\n",
    "labels_placeholder = tf.placeholder(tf.int64, [None, ])\n",
    "dataset = make_dataset(images_placeholder, labels_placeholder,\n",
    "                       epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "# 定义迭代器\n",
    "dataset_iter = dataset.make_initializable_iterator()\n",
    "x, y = dataset_iter.get_next()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    # 初始化\n",
    "    sess.run(dataset_iter.initializer, feed_dict = {\n",
    "        images_placeholder: x_train_scaled,\n",
    "        labels_placeholder: y_train\n",
    "    })\n",
    "    x_val, y_val = sess.run([x, y])\n",
    "    print(x_val.shape)\n",
    "    print(y_val.shape)\n",
    "    sess.run(dataset_iter.initializer, feed_dict = {\n",
    "        images_placeholder: x_valid_scaled,\n",
    "        labels_placeholder: y_valid\n",
    "    })\n",
    "    x_val, y_val = sess.run([x, y])\n",
    "    print(x_val.shape)\n",
    "    print(y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
